#!/bin/bash
# Author :David 
# Usage  :POX back to HAS
# Date   :201304 create script  -David
#        :20130626 add function (sync changed table)  -David
# History:
#        07/22/2013,Leslie Zhu, czhu@kdsglobal.com
#        after udpate pox-HAS-map, begin sync data to N env
#				 07/30/2013,David,sync to nfs mount point
#
#

mailto="wzhao@kdsglobal.com,czhu@kdsglobal.com,cliu@kdsglobal.com,mli@kdsglobal.com,dwang@kdsglobal.com,hzhang@kdsglobal.com,jshi@kdsglobal.com,alee@kdsglobal.com,lthoman@kdsglobal.com"

mailtest=dwang@kdsglobal.com

backup_pat='fh|fn|g[12n]|empty|cmo|hp|Issr|CMO|state|hets'

function pushcode (){
	local xnode=$1
	rsync -a UBBack2HAS.sh pox-HAS-map backupdirs.lst tb.lst  x$xnode:~/bin/HAS
}

function gen_new_tb_lst(){
	day=$1
	>/tmp/tb.lst
	for i in {1..36}
	do
		for table in $(find /UBSys/NodesPOD/$i/tables/ -mtime -$day)
		do
			grep "dir" $table |awk -F" = " '{print $2}'  >>/tmp/tb.lst
		done
	done
	sort -u /tmp/tb.lst >tb.lst
}

function ck-p-rate(){
	dirs=$(wc -l backupdirs.lst|awk '{print $1}')
	wc -l	/UBSys/NodesPOD/*/.backupHAS/$1*.sync.log |tail -1 | awk -v dirc=$dirs '{print $1/(36*dirc)/2*100"%"}'
}


function ck-node(){
	dirs=$(wc -l backupdirs.lst|awk '{print $1}')
	for node in {1..36};do
		wc -l /UBSys/NodesPOD/$node/.backupHAS/$1.$node.sync.log |tail -1|awk -v dirc=$dirs '{print $1/dirc/2*100"%"}'
	done

}

function list_disks(){
	node=$1
	[ -n "$node" ] || node=1
	for disk in  $(ls /UBSys/NodesPOD/$node/Disks/ 2> /dev/null);
  	do
    	if [ $disk -ge 0 ] 2> /dev/null ;then echo $disk" ";fi
  	done
}

function list_disk_dirs(){
	node=$1
	disks=$(list_disks $node)
	for d in $disks
	do
		for i in $(ls -1 /UBSys/NodesPOD/$node/Disks/$d); do
			echo $i |egrep -sq '\-tmp\-' && continue
      		[ -d /UBSys/NodesPOD/$node/Disks/$d/$i ] && echo Disks/$d/$i
  		done
	done
}

function gen_backupdirs(){
	cd ~/bin/HAS
	for i in {1..36};do
		./UBBack2HAS.sh list_disk_dirs $i >tmp/dirs.$i
	done
	dir=$(wc -l tmp/dirs.* |sort -n |head -1|awk '{print $2}')
	egrep "$backup_pat" $dir
}

function gen-table-list () {
	args=$1
	echo $args
	cat /UBSys/NodesPOD/1/db.ini |grep 'location =' |awk -F"= " '{print $2}' |grep $args >table.lst
}



function has-xmap(){
	local hasID=$1
	nodes=$(egrep -w has$hasID pox-HAS-map|cut -f1|cut -c 2-)
	echo $nodes	
}

function sync-has (){
	local hasID=$1
	pushcode $hasID
	ssh has$hasID "cd POX;./UBBack2HAS.sh sync-has-local $hasID >> ~/POX/UBBack.log &"
}

function sync-xnode (){
	local xnode=$1
	pushcode $xnode
	ssh x$xnode "cd bin/HAS; ./UBBack2HAS.sh sync-xnode-local $xnode >logs/UBBack.log & "
}

function sync-node-local (){
	node=$1
  ndir=/UBSys/NodesPOD
	#dstdir=/HAS/POD1/UBSys/HASNodesNpair/
	dstdir=/HAS/POD1/UBSys/HASNodes/
	#dstpdir=/HAS/POD1/UBSys/HASNodesNpair/
	cmodst=/HAS/POD1_H101/TNodesCMO/ 
	#dstdir=/HAS/POD1/UBSys/HASNodesNP1/
	dt=$(date +"%Y%m%d")
	errlog=$ndir/$node/.backupHAS/$dt.$node.err.log
	synclog=$ndir/$node/.backupHAS/$dt.$node.sync.log
	echo ============Node $node $dt sync err log============ > $errlog
  echo $(date +"%Y%m%d %T") Start Rsync Node $node > $synclog
	touch $ndir/$node/.backupHAS/$dt.$node.running
	for dir in $(<backupdirs.lst)
	do
		echo  $(date +"%Y%m%d %T") Start Rsync $dir >>$synclog
		rsync -aL --delete  --exclude ".*" $ndir/$node/$dir/ $dstdir/$node/$dir/ >> $errlog  2>&1
	#	rsync -aL --delete  --exclude ".*" $ndir/$node/$dir/ $dstpdir/$node/$dir/ >> $errlog  2>&1
		echo $(date +"%Y%m%d %T") End"   "Rsync $dir >> $synclog 
	done
	echo  $(date +"%Y%m%d %T") Start Rsync Others >>$synclog
	rsync -aL --exclude "Disks" $ndir/$node/ $dstdir/$node/ >>$errlog 2>&1
#	rsync -aL --exclude "Disks" $ndir/$node/ $dstpdir/$node/ >>$errlog 2>&1
	rsync -a $ndir/$node/tables/ $cmodst/$node/tables/
	echo $(date +"%Y%m%d %T") End"   "Rsync Others >> $synclog
	mv $ndir/$node/.backupHAS/$dt.$node.running $ndir/$node/.backupHAS/$dt.$node.end
	touch $ndir/$node/.backupHAS/$dt.$node.end
	echo $(date +"%Y%m%d %T") End"   "Rsync Node $node >> $synclog
}


function sync-node-local-old(){
	xnode=$1
	node=$2
	moaip=$(egrep  x$xnode moa.map |cut -d" " -f2)
	hdir=/mnt/z0/POD1
	ndir=/UBSys/NodesPOD
	#logf=$(date +"%Y%m%d")
	dt=$(date +"%Y%m%d")
	logf=~/POX/logs/$dt.$node.err.log
	logr=~/POX/logs/$dt.$node.sync.log
	mkdir -p ~/POX/logs/$node
	#touch ~/POX/logs/$node.$dt.start
	echo ============Node $node $dt sync err log============ > $logf
	echo $(date +"%Y%m%d %T") Start Rsync Node $node > $logr
	touch ~/POX/logs/$node/$dt.$node.running
	ssh x22 "touch ~/HAS/logs/$dt.$node.running"
	for dir in $(<backupdirs.lst)
	do
		mkdir -p $hdir/$node/$dir
		echo $(date +"%Y%m%d %T") Start Rsync $dir >> $logr
		rsync -aL --delete --exclude ".*"  $moaip:$ndir/$node/$dir/ $hdir/$node/$dir/  >> $logf 2>&1
		echo $(date +"%Y%m%d %T") End"   "Rsync $dir >> $logr
	done
	rsync -aL --exclude "Disks"  $moaip:$ndir/$node/ $hdir/$node/  >>$logf 2>&1
	mv  ~/POX/logs/$dt.$node.running   ~/POX/logs/$node/$dt.end
	touch ~/POX/logs/$node/$dt.end
	ssh x22 "mv ~/HAS/logs/$dt.$node.running ~/HAS/logs/$dt.$node.end ;touch ~/HAS/logs/$dt.$node.end"
	#errc=$(wc -l $logf |cut -d" " -f1)
	#if [ "$errc" -gt 1 ] ;then
	#	mail -s "POX Daily Backup Err Msg Of Node $node" $mailtest < $logf
	#fi
	echo $(date +"%Y%m%d %T") End"   "Rsync Node $node >> $logr
	#scp $logf x22:~/HAS/logs  
}

function sync-xnode-local(){
	local xnode=$1
	cd ~/bin/HAS
	node1=$((($1-23)*2+1))
	node2=$((($1-23)*2+2))
	for n in $node1 $node2
	do
		sync-node-local $n >/dev/null  &
	done
}

function sync-has-local(){
	local hasID=$1
	cd ~/POX
	nodes=$(has-xmap $hasID)
	#echo $nodes >test.txt
	for i in $nodes
	do
		xnode=$(((i-1)/2+23))
		sync-node-local $xnode $i & 
	done
}


function sync-static(){
	for i in {23..40}
	do
		rsync -a  UBBack2HAS.sh tb.lst x$i:~/bin/HAS/
		ssh x$i "cd bin/HAS; ./UBBack2HAS.sh sync-static-xnode $i >/dev/null &"
	done
}

function sync-static-xnode () {
	xnode=$1
	n1=$(((xnode-23)*2+1))
	n2=$(((xnode-23)*2+2))
	for n in $n1 $n2 
	do
		sync-static-node-local $n  >~/bin/HAS/test.log 2>&1 &
	done
}

function sync-static-node-local() {
	node=$1
	toppath=/HAS/POD1_H101/TNodes
	#toppath=/HAS/POD1_H101/TNodesMP
	#toppath=/HAS/POD1/UBSys/HASNodes
	#toppath=/HAS/POD1_H101/TNodesRP/
	#toppath=/HAS/POD1_H101/TNodesCMO
	cd ~/bin/HAS/
	cat tb.lst |while read path
	do
		#mkdir -p $toppath/$node/$path 
		#rsync -a $toppath/$node/$path/ /UBSys/NodesPOD/$node/$path/ >> ~/bin/HAS/static.log 2>&1
		rsync -a /UBSys/NodesPOD/$node/$path/ $toppath/$node/$path/  >> ~/bin/HAS/static.log 2>&1
		#rsync -a $toppath/$node/$path/ /HAS/POD1/UBSys/HASNodesNP1/$node/$path/ >> ~/bin/HAS/static.log 2>&1
		#rsync -a /UBSys/NodesPOD/$node/$path/ $toppath/$node/$path/ >> ~/bin/HAS/static.log 2>&1
		#rsync -a /UBSys/NodesPOD/$node/$path/ /HAS/POD1/UBSys/HASNodesNP1/$node/$path/ >> ~/bin/HAS/static.log 2>&1 
	done
	#rsync -a /UBSys/NodesPOD/$node/tables/ $toppath/$node/tables/ >> ~/bin/HAS/static.log 2>&1
}

function daily_backup(){
	cd ~/bin/HAS
	dt=$(date +"%Y%m%d")
	#args=$(date +"%Y%m")
	#gen-table-list $args
	#cat /UBSys/NodesPOD/1/db.ini |grep 'location =' |awk -F"= " '{print $2}' |grep static >>table.lst
	gen_backupdirs >backupdirs.lst
	for i in {23..40}
	do
		pushcode $i
		sync-xnode $i 
	done
	get_sync_log $dt 
}

function ck-running (){
	local dt=$1
	rcount=0
	for i in {1..36}
	do	
		node=$i
		if [ -f /UBSys/NodesPOD/$node/.backupHAS/$dt.$node.running ] ;then 
			((rcount++))
		#	echo $i running $rcount
		fi
	done
	echo $rcount
}

function scan-err (){
	dt=$1
	for i in {1..36}
	do
		linec=$(wc -l /UBSys/NodesPOD/$i/.backupHAS/$dt.$i.err.log |awk '{print $1}')
		if [ ! "$linec" -eq 1 ]; then 
		 cat /UBSys/NodesPOD/$i/.backupHAS/$dt.$i.err.log  
			echo
		
		fi 
	done
}

function get_sync_time(){
	local dt=$1
	echo -e Node"      "HAS "       "Start"     "End"         "Cost
	for i in {1..36} ;do
		hasID=$(head -$i pox-HAS-map |tail -1|cut -f2)
		stime=$(head -1 /UBSys/NodesPOD/$i/.backupHAS/$dt.$i.sync.log  |cut -d" " -f2)
		etime=$(tail -1 /UBSys/NodesPOD/$i/.backupHAS/$dt.$i.sync.log  |cut -d" " -f2)
		ss=$(date -d "$dt $stime" +%s)
		se=$(date -d "$dt $etime" +%s)
		cost=$(($se-$ss))
		bt=$(date -d "010101 $cost seconds" +%H:%M:%S)
		xnode=$((($i-1)/2+23))
		echo -e  x$xnode-$i"     "$hasID"    "$stime"  "$etime"   "$bt
	done
	
}

function ck-node-status(){
	local dt=$1
	dirs=$(wc -l backupdirs.lst|awk '{print $1}')
	echo -e Nodes"    "HAS "     "isRunning "    "Pct
	for i in {1..36} ; do
	hasID=$(head -$i pox-HAS-map |tail -1|cut -f2)
	if [ -f /UBSys/NodesPOD/$i/.backupHAS/$dt.$i.running ] ; then
		isRun=Yes
	else 
		isRun=No
	fi
	line=$(wc -l /UBSys/NodesPOD/$i/.backupHAS/$dt.$i.sync.log |cut -f1 )
	pct=$(echo $line |awk -v ls=$dirs '{printf("%.2f",$1/(ls*2)*100)}')
	xnode=$((($i-1)/2+23))
	echo x$xnode-$i "   "$hasID"    " $isRun "       "$pct
	done
}
	
function get_sync_log() {
	dt=$1
	rcount=$(ck-running $dt)
	while [ "$rcount" -ne 0 ]
	do
		sleep 600
		rcount=$(ck-running $dt)
	done
	get_sync_time $dt >/tmp/$dt.sync.log
	scan-err $dt >>/tmp/$dt.sync.log
	mail -s "POX To HAS Daily Backup Log $dt" $mailto < /tmp/$dt.sync.log

}

function get_sync_logtest() {
  dt=$1
  rcount=$(ck-running $dt)
  while [ "$rcount" -ne 0 ]
  do
    sleep 600
    rcount=$(ck-running $dt)
  done
  get_sync_time $dt >/tmp/$dt.sync.log
  scan-err $dt >>/tmp/$dt.sync.log
  mail -s "POX To HAS Daily Backup Log $dt" $mailtest < /tmp/$dt.sync.log
}

function get_node_time(){
	echo $1 $2 $3
	dt=$1
	node=$2
	hasID=$3
	stime=$(ssh $hasID "head -1 ~/POX/logs/$dt.$node.sync.log" |awk -F"Start" '{print $1}')
	etime=$(ssh $hasID "tail -1 ~/POX/logs/$dt.$node.sync.log" |awk -F"End" '{print $1}')
	echo Node $node $stime $etime
}


function sync-table (){
	[ -z "$1" ] && {
		echo pls input filter conditions
		exit 
	}
	gen-table-list $1
	for i in {101..111}
    	do
        	sync-has $i
    	done

}

function has2node() { # has env: display node numbers for has in env
	  local has=$1
    local env=$2
    hasmap=/UBSys/local/etc/auto.HAS
    for i in $(grep has$has $hasmap |egrep ^${env}_ |awk '{print $2}'); do
        basename $i
    done
}

function node2has() { # env_node;display hasnumber for node in env
	local node=$1
	hasmap=/UBSys/local/etc/auto.HAS
	hasmap2=/UBSys/local/etc/auto.HAS2
	has=$(grep -P  "^$node\t"   $hasmap|cut -f2 |cut -c 1-6)
	echo $has
}

function failover-status() {
	hasmap=/UBSys/local/etc/auto.HAS
	hasmap2=/UBSys/local/etc/auto.HAS2
	\ls -l /HAS/POD1/UBSys/HASNodes/ |grep HAS2 |awk -F"/" '{print $NF}' |while read node
	do
		has=$(grep -P  "^$node\t"   $hasmap|cut -f2 |cut -c 1-6)
		pairhas=$(grep -P  "^$node\t"   $hasmap2|cut -f2 |cut -c 1-6)
		echo $has "->" $pairhas	
	done |sort -u 
}

function gen-pairsync-cmd () { # env ;generate pair-sync cmd
	env=$1
	hasmap=/UBSys/local/etc/auto.HAS
  hasmap2=/UBSys/local/etc/auto.HAS2
	failover-status|cut -c 1-6 >/HAS/POD1/UBSys/HASNodes/.failover_status

	cat $hasmap|sort  >/tmp/map1
	cat $hasmap2 |sort >/tmp/map2
	join /tmp/map2 /tmp/map1 |grep "${env}_" |sort -k2n -t_ |tr ":" " " |awk '
	BEGIN{
		while(getline <"/HAS/POD1/UBSys/HASNodes/.failover_status") fail[$1]=1;
		close("/HAS/POD1/UBSys/HASNodes/.failover_status");
	}
	{
		dt=strftime("%Y%m%d",systime());
		if(($2 in fail) || ($4 in fail))
			print "#",$2 ,"or " $4, "in failover mode,skip this pair";
		else
			print "ssh",$2,"\"rsync -avx" ,$4":"$3"/",$3"/" ,">"$3"/.pairBackup/"dt".log 2>&1 &" "\""
	}' |tee /HAS/POD1/UBSys/HASNodes/.pairsync-cmd
	bash /HAS/POD1/UBSys/HASNodes/.pairsync-cmd
}

eval $*
